# /home/node54_tmpdata/xlgeng/LibriSpeech/train-clean-360
#!/bin/bash

# Copyright 2019 Mobvoi Inc. All Rights Reserved.

. ./path.sh || exit 1;



# bpemode (unigram or bpe)
nbpe=1000
bpemode=unigram

set -e
set -u
set -o pipefail

stage=1
stop_stage=1
. tools/parse_options.sh || exit 1

dict=data/lang_char/${bpemode}${nbpe}_units.txt
bpemodel=data/lang_char/${bpemode}${nbpe}
text_path=/home/node54_tmpdata/xlgeng/LibriSpeech/train-clean-360/train-clean-360.trans
echo "dictionary: ${dict}"
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
  ### Task dependent. You have to check non-linguistic symbols used in the corpus.
  echo "stage 2: Dictionary and Json Data Preparation"
  mkdir -p data/lang_char/

  echo "<blank> 0" > ${dict} # 0 will be used for "blank" in CTC
  echo "<unk> 1" >> ${dict} # <unk> must be 1
  echo "<sos/eos> 2" >> $dict # <eos>

  # we borrowed these code and scripts which are related bpe from ESPnet.
  cut -f 2- -d" " $text_path > data/lang_char/input.txt
  tools/spm_train --input=data/lang_char/input.txt --vocab_size=${nbpe} --model_type=${bpemode} --model_prefix=${bpemodel} --input_sentence_size=100000000
  tools/spm_encode --model=${bpemodel}.model --output_format=piece < data/lang_char/input.txt | tr ' ' '\n' | sort | uniq | awk '{print $0 " " NR+2}' >> ${dict}
fi




